#!/usr/bin/env bash
set -ex

command -v nvidia-smi >/dev/null || exit 0

error() {
    logger -s -t slurm "Draining node -- $1"
    scontrol update nodename="$HOSTNAME" state=drain reason="$1"
    exit 1
}

if ! {{ slurm_sysconf_dir }}/shared/bin/set_gpu_power_levels.sh max >/dev/null ; then
    error 'Failed to set GPU power levels'
fi
if ! {{ slurm_sysconf_dir }}/shared/bin/set_gpu_clocks.sh max >/dev/null ; then
    error 'Failed to set GPU clocks'
fi
